#!/usr/bin/env python
# coding=utf-8
from bs4 import BeautifulSoup
import re

'''
参考网页
https://www.w3school.com.cn/cssref/css_selectors.asp    css选择器
'''
html = """
<html><head><title>The Dormouse's story</title></head>
<body>
<p class="title" name="dromouse"><b>The Dormouse's story</b></p>
<p class="story">Once upon a time there were three little sisters; and their names were
<a href="http://example.com/elsie" class="sister" id="link1"><!-- Elsie --></a>,
<a href="http://example.com/lacie" class="sister" id="link2">Lacie</a> and
<a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>;
and they lived at the bottom of a well.</p>
<p class="story">...</p>
"""

# 4.基本用法
soup = BeautifulSoup(html, 'lxml')  # 初始化html文本
print(soup.prettify())  # 按标准html格式输出
print(soup.title.string)  # 提取title节点的文本

# 5.节点选择器
print('\n')
print("5.节点选择器")
print('--选择元素')
print(soup.title)
print(type(soup.title))
print(soup.title.string)
print(soup.head)
print(soup.p)
print('--提取信息')
print('(1)name--获取属性节点的名称')
print(soup.title.name)
print('(2)attrs--获取属性')
print(soup.p.attrs)  # 返回字典类型
print(soup.p.attrs['name'])
print(soup.p['name'])  # 节点元素后面加[属性名],可获得属性值
print(soup.p['class'])
print('(3).string--获取节点元素包含的文本内容')
print(soup.p.string)
print('--嵌套选择')
html2 = '''
<html><head><title>The Dormouse's story</title></head>
<body>
'''
soup = BeautifulSoup(html2, 'lxml')
print(soup.head.title)
print(type(soup.head.title))
print(soup.head.title.string)
print('--关联选择')
print('(1)子节点和子孙节点')
html3 = """
<html>
    <head>
        <title>The Dormouse's story</title>
    </head>
    <body>
        <p class="story">
            Once upon a time there were three little sisters; and their names were
            <a href="http://example.com/elsie" class="sister" id="link1">
                <span>Elsie</span>
            </a>
            <a href="http://example.com/lacie" class="sister" id="link2">Lacie</a> 
            and
            <a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>
            and they lived at the bottom of a well.
        </p>
        <p class="story">...</p>
"""
soup = BeautifulSoup(html3, 'lxml')
print('content--获取节点元素的直接子节点')
print(soup.p.contents)  # content--获取节点元素的直接子节点的列表
print('children--同样返回子节点,返回结果为生成器类型')
print(soup.p.children)
for i, child in enumerate(soup.p.children):
    print(i, child)
print('descendants--返回子孙节点,返回结果为生成器类型')
print(soup.p.children)
for i, child in enumerate(soup.p.descendants):
    print(i, child)
print('(2)父节点和祖先节点,parent属性获得直接父节点')
html4 = """
<html>
    <head>
        <title>The Dormouse's story</title>
    </head>
    <body>
        <p class="story">
            Once upon a time there were three little sisters; and their names were
            <a href="http://example.com/elsie" class="sister" id="link1">
                <span>Elsie</span>
            </a>
        </p>
        <p class="story">...</p>
"""
soup = BeautifulSoup(html4, 'lxml')
print(soup.a.parent)
print('获得所有祖先节点')
print(list(enumerate(soup.a.parents)))
print('(3)兄弟节点')
html = """
<html>
    <body>
        <p class="story">
            Once upon a time there were three little sisters; and their names were
            <a href="http://example.com/elsie" class="sister" id="link1">
                <span>Elsie</span>
            </a>
            Hello
            <a href="http://example.com/lacie" class="sister" id="link2">Lacie</a> 
            and
            <a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>
            and they lived at the bottom of a well.
        </p>
"""
soup = BeautifulSoup(html, 'lxml')
print('Next Sibling(下一个兄弟节点)', soup.a.next_sibling)
print('Prev Sibling(上一个兄弟节点)', soup.a.previous_sibling)
print('Next Sibling(下一个兄弟节点列表)', list(enumerate(soup.a.next_siblings)))
print('Prev Sibling(上一个兄弟节点列表)', list(enumerate(soup.a.previous_siblings)))
print('(4)提取信息')
html = """
<html>
    <body>
        <p class="story">
            Once upon a time there were three little sisters; and their names were
            <a href="http://example.com/elsie" class="sister" id="link1">Bob</a>
            <a href="http://example.com/lacie" class="sister" id="link2">Lacie</a> 
        </p>
"""
soup = BeautifulSoup(html, 'lxml')
print('下一个兄弟节点:')
print(type(soup.a.next_sibling))
print(soup.a.next_sibling)
print(soup.a.next_sibling.string)
print('父节点:')
print(type(soup.a.parents))
print(list(soup.a.parents)[0])
print(list(soup.a.parents)[0].attrs['class'])

# 6.方法选择器
# --find_all()
# (1) name
html = '''
<div class="panel">
    <div class="panel-heading">
        <h4>Hello</h4>
    </div>
    <div class="panel-body">
        <ul class="list" id="list-1">
            <li class="element">Foo</li>
            <li class="element">Bar</li>
            <li class="element">Jay</li>
        </ul>
        <ul class="list list-small" id="list-2">
            <li class="element">Foo</li>
            <li class="element">Bar</li>
        </ul>
    </div>
</div>
'''
soup = BeautifulSoup(html, 'lxml')
print('find_all--name获取')
print(soup.find_all(name='ul'))
print(type(soup.find_all(name='ul')[0]))
print(soup.find_all(name='ul')[1])  # diy
for ul in soup.find_all(name='ul'):  # 因为是嵌套,所以用两次循环获取最终节点的文本
    print(ul.find_all(name='li'))
    for li in ul.find_all(name='li'):
        print(li.string)

# (2) attrs  根据属性查询
html = '''
<div class="panel">
    <div class="panel-heading">
        <h4>Hello</h4>
    </div>
    <div class="panel-body">
        <ul class="list" id="list-1" name="elements">
            <li class="element">Foo</li>
            <li class="element">Bar</li>
            <li class="element">Jay</li>
        </ul>
        <ul class="list list-small" id="list-2">
            <li class="element">Foo</li>
            <li class="element">Bar</li>
        </ul>
    </div>
</div>
'''
soup = BeautifulSoup(html, 'lxml')
print('根据属性查询')
print(soup.find_all(attrs={'id': 'list-2'}))
print(soup.find_all(attrs={'name': 'elements'}))
print('常用文本id,class等,直接传入')
print(soup.find_all(id='list-1'))
print(soup.find_all(class_='element'))
# (3) text--匹配节点的文本
html = '''
<div class="panel">
    <div class="panel-body">
        <a>Hello, this is a link</a>
        <ul><li>this</li</ul>
        <a>Hello, this is a link, too</a>
    </div>
</div>
'''
soup = BeautifulSoup(html, 'lxml')
print(soup.find_all(text=re.compile('this')))
# --find 匹配第一个元素
html = '''
<div class="panel">
    <div class="panel-heading">
        <h4>Hello</h4>
    </div>
    <div class="panel-body">
        <ul class="list" id="list-1">
            <li class="element">Foo</li>
            <li class="element">Bar</li>
            <li class="element">Jay</li>
        </ul>
        <ul class="list list-small" id="list-2">
            <li class="element">Foo</li>
            <li class="element">Bar</li>
        </ul>
    </div>
</div>
'''
print('find()方法')
soup = BeautifulSoup(html, 'lxml')
print(soup.find(name='ul'))
print(type(soup.find(name='ul')))
print(soup.find(class_='list'))
'''
soup.find_parent()  返回直接父节点
soup.find_parents() 返回所有祖先节点
soup.find_next_sibling()    返回前面第一个兄弟节点
soup.find_next_siblings()   返回前面所有兄弟节点
soup.find_next()    返回第一个符合条件的节点
soup.find_all_next()    返回节点后所有符合条件的节点    
soup.find_previous()    返回第一个符合条件的节点
soup.find_all_previous()    返回节点后所有符合条件的节点
'''
# 7. css选择器
html = '''
<div class="panel">
    <div class="panel-heading">
        <h4>Hello</h4>
    </div>
    <div class="panel-body">
        <ul class="list" id="list-1">
            <li class="element">Foo</li>
            <li class="element">Bar</li>
            <li class="element">Jay</li>
        </ul>
        <ul class="list list-small" id="list-2">
            <li class="element">Foo</li>
            <li class="element">Bar</li>
        </ul>
    </div>
</div>
'''
print('css选择器')
soup = BeautifulSoup(html, 'lxml')
print(soup.select('.panel .panel-heading'))
print(soup.select('ul li'))
print(soup.select('#list-2 .element'))
print(type(soup.select('ul')[0]))
# --嵌套选择
print('嵌套选择')
for ul in soup.select('ul'):
    print(ul.select('li'))
# --获取属性
print('获取属性')
for ul in soup.select('ul'):
    print(ul['id'])
    print('attrs:' + ul.attrs['id'])
# --获取文本
print('获取文本')
for li in soup.select('li'):
    print('Get Text:',li.get_text())
    print('String:',li.string)